#ifndef CSE220_project3_Tokenizer_h
#define CSE220_project3_Tokenizer_h
using namespace std;

#define ARRAY_SIZE(array) (sizeof((array))/sizeof((array[0])))

#include "common.h"
#include "Literal.h"
#include "Scanner.h"

class Scanner;

class Tokenizer
{
public:    
    Tokenizer(Scanner &);
    bool FetchToken();
    string GetTokenCode();
    string GetTokenString();
    Literal *FetchLiteral();
    ~Tokenizer();
    
private:
    string code;
    string tokenString;
    Literal *lit;
    Scanner *scanner;
    
    char SkipBlanks();
    void FetchWord(char, CharCode);
    void FetchNumber(char, CharCode);
    void FetchString(char);
    void FetchSpecial(char);
    bool IsReservedWord(string &);
};

#endif
